#define _GNU_SOURCE

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <sys/ioctl.h>
#include <asm/unistd.h>
#include <xmmintrin.h>      /* _mm_clflush */
#include <sched.h>
#include <unistd.h>         /* sysconf */
#include <stdio.h>
#include <sys/wait.h>
#include <sys/mman.h>

#define TYPE unsigned long  
#define arr_size_X  13107200  // 100mb 
#define arr_size_Y  1

typedef unsigned long __u64;
typedef unsigned int __u32;

static inline __u64 rdtscp()
{
    __u32 lo,hi;

    __asm__ __volatile__
    (
        "rdtscp":"=a"(lo),"=d"(hi)
    );
    return (__u64)hi<<32|lo;
}

void * smem();

void myinit(TYPE *mem, unsigned long step_size){
    int i, j;
    TYPE *mem1;
    TYPE *mem_end = mem + (arr_size_X * arr_size_Y);

    j = 0;
    for (i=0; i < arr_size_Y; i++)
        for (mem1 = mem + i; mem1 < mem_end; mem1+=arr_size_Y)
        {
            *mem1 = 0;
        }

    for (i=0; i < arr_size_Y; i++)
        for (mem1 = mem + i; mem1 < mem_end; mem1+=arr_size_Y){
            _mm_clflush(mem1);
        }
    return ;
}

int work(TYPE *mem, unsigned long *control_number)
{
    __u64 begin = 0, end = 0, sum_tsc = 0, max_tsc = 0, min_tsc = 0x7fffffffffffffff;

    unsigned long i, ssize = 0;
    int k = 0;
    TYPE *mem1;
    int cur_cpu = 2;

    unsigned long step_size = (unsigned long) control_number[1];
    unsigned long loop_number_1 = (unsigned long) control_number[2];  //外层循环
    unsigned long loop_number_2 = (unsigned long) 2;      //内层循环


    cur_cpu = sched_getcpu();
    printf("-----------SUB Process at CPU: %d----------\n", cur_cpu);       

    //起始偏移 循环次数 
    for(i=0; i<=loop_number_1; i++)
    {
        ssize = i * step_size;
        sum_tsc = 0;
        max_tsc = 0;
        min_tsc = 0x7fffffffffffffff;
        begin = rdtscp();
        __asm__ __volatile__
        (
            "mov %2, %%r13\n\t"         // 增加一个循环，两层循环。定为100
            "lea %0, %%r8\n\t"          // 内存块开始地址
            "TOP_LOOP:\n\t"
                "mov $256, %%rax\n\t"
                "mull %%r13d\n\t"
                "add %%rax, %%r8\n\t"
                "mov %3, %%r12\n\t"         // 循环次数 如果为2，就是一对地址。这里将保持为2。
                "LOOP1:\n\t"
                    "mov (%%r8), %%r14\n\t"    // 读一对地址中的A
                    "add %1, %%r14\n\t"        // 读取A的结果，加 i * 64
                    "add %%r14, %%r8\n\t"      // 得到一对地址中的下一个地址：B。（依赖）
                "sub $1, %%r12\n\t"        // 循环
                "jne LOOP1\n\t"            // 向上跳转
                "sub %%r14, %%r8\n\t"      // 读取A的结果，加 i * 64
                "sub %%r14, %%r8\n\t"      // 读取A的结果，加 i * 64
                "sub %%rax, %%r8\n\t"      // 以上三行迷惑CPU，在地址对间产生依赖
            "sub $1, %%r13\n\t"
            "jne TOP_LOOP\n\t"
            :
            :"m"( *mem ),"r"(ssize),"i"(100),"r"(loop_number_2)        //立即数1：循环次数  立即数2：步进
            :"rax","rdx","r8","r12","r13","r14"
        );
        end = rdtscp();
        end = end - begin;
        sum_tsc += end;
        max_tsc = (max_tsc > end ? max_tsc : end);
        min_tsc = (min_tsc < end ? min_tsc : end);                

        fprintf(stderr,  "ssize=%15ld ", ssize);
        fprintf(stderr,  "CPU:%d:%12ld %12ld% 12ld\n", cur_cpu, sum_tsc, min_tsc, max_tsc);

        __asm__ __volatile__
        (
            "mov %2, %%r13\n\t"         //增加一个循环，两层循环。定为1000
            "lea %0, %%r8\n\t"          // 内存块开始地址
            "TOP_LOOP_CR:\n\t"
                "mov $256, %%rax\n\t"
                "mull %%r13d\n\t"
                "add %%rax, %%r8\n\t"
                "mov %3, %%r12\n\t"         // 循环次数 如果为2，就是一对地址
                "LOOP1_CR:\n\t"
                    "mov (%%r8), %%r14\n\t"    // 读一对地址中的A
                    "clflush (%%r8)\n\t"       // 刷新L1~L3 Cache
                    "add %1, %%r14\n\t"        // 读取A的结果，加 i * 64
                    "add %%r14, %%r8\n\t"      // 得到一对地址中的下一个地址：B。（依赖）
                "sub $1, %%r12\n\t"        // 循环
                "jne LOOP1_CR\n\t"            // 向上跳转
                "sub %%r14, %%r8\n\t"         // 读取A的结果，加 i * 64
                "sub %%r14, %%r8\n\t"         // 读取A的结果，加 i * 64
                "sub %%rax, %%r8\n\t"
            "sub $1, %%r13\n\t"
            "jne TOP_LOOP_CR\n\t"
            :
            :"m"( *mem ),"r"(ssize),"i"(100),"r"(loop_number_2)        //立即数1：循环次数  立即数2：步进
            :"rax","r8","r9","r10","r11","r12","r13","r14","r15"
        );    
    }

    return k;
}

void * smem()
{   
    void *ptr;
    ptr = mmap(0, arr_size_X * arr_size_Y * sizeof(TYPE), PROT_READ | PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS|0x40000, -1, 0); // 大页
    // ptr = mmap(0, arr_size_X * arr_size_Y * sizeof(TYPE), PROT_READ | PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0);   // 普通页
    if (ptr == MAP_FAILED)
        perror("Failed to allocate shared memory\n");
    return ptr;
}

void setcpu(int cpu)
{
    cpu_set_t mask;

    CPU_ZERO(&mask);
    CPU_SET(cpu, &mask);

    if (sched_setaffinity(0, sizeof(cpu_set_t), &mask) == -1) 
    {   
        perror("sched_setaffinity fail!");
        exit(EXIT_FAILURE);
    }
}

void main(int argc, char **argv) {
    TYPE *mem;
    int k, t, p_stat, cur_cpu, start_cpu_id;
    unsigned long control_number[3];
    pid_t pid[8];

    if (argc < 5) {
        fprintf(stderr, "numa start_cpu_id run_cpu_id step_size loop_number\n");
        return ;
    }

    start_cpu_id = atoi(argv[1]);
    control_number[0] = atoi(argv[2]);          // RUN CPU ID
    control_number[1] = atoi(argv[3]);          // 步幅
    control_number[2] = atoi(argv[4]);          // 外循环次数  （内循环次数固定为2）

    cur_cpu = sched_getcpu();
    printf("Old CPU: %d\n", cur_cpu);       
    setcpu(start_cpu_id);                             // 按参数指定CPU运行
    cur_cpu = sched_getcpu();
    printf("Current CPU: %d\n", cur_cpu);       

    mem=(TYPE *)smem();
    if ( mem == MAP_FAILED )
    {
        printf("Failed to allocate shared memory\n");
        return ;
    }

    myinit(mem, control_number[1]);
    setcpu(control_number[0]);          // control_number[0] : 运行CPU

    for (t=0; t<1; t++)
    {
        pid[t] = fork();
        if (pid[t] == 0)                // child process
        {
            k = work(mem, &control_number[0]);
            return ;
        }
        else if ( pid[t] < 0 )
            printf("fork error\n");
    }

    printf("Parent PID:%d  Child PID:%d\n", getpid(), pid[0]);
    for (t=0; t<1 ; t++)
    {
        if ( pid[t] > 0 )
        {
            printf("PID is %d \n", pid[t]);
            waitpid(pid[t], &p_stat, 0);
        }
        else if ( pid[t] < 0 )
            printf("fork error\n");
        printf("============================\n");
    }
    munmap(mem, arr_size_X * arr_size_Y * sizeof(TYPE)); 
    return ;
}